{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. 调整学习率及L2正则参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import argparse\n",
    "import sys\n",
    "import tensorflow as tf\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "\n",
    "\n",
    "from keras.layers.core import Dense, Flatten\n",
    "from keras.layers.convolutional import Conv2D\n",
    "from keras.layers.pooling import MaxPooling2D\n",
    "\n",
    "from keras import backend as K\n",
    "import time\n",
    "from keras import initializers\n",
    "from keras.objectives import categorical_crossentropy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /tmp/tensorflow/mnist/input_data\\train-images-idx3-ubyte.gz\n",
      "Extracting /tmp/tensorflow/mnist/input_data\\train-labels-idx1-ubyte.gz\n",
      "Extracting /tmp/tensorflow/mnist/input_data\\t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/tensorflow/mnist/input_data\\t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "# Import data\n",
    "data_dir = '/tmp/tensorflow/mnist/input_data'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define loss and optimizer\n",
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "learning_rate = tf.placeholder(tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def CONV(lr = 0.01, Lambda = 7e-5):\n",
    "    with tf.name_scope('reshape'):\n",
    "            x_image = tf.reshape(x, [-1, 28, 28, 1])\n",
    "\n",
    "    net = Conv2D(32, kernel_size=[5,5], strides=[1,1],activation='relu', \n",
    "                     padding='same',\n",
    "                     kernel_initializer = initializers.TruncatedNormal(stddev=0.1), #卷积权重初始化\n",
    "                     bias_initializer = initializers.Zeros(), #偏置权重初始化\n",
    "                    input_shape=[28,28,1])(x_image)\n",
    "    net = MaxPooling2D(pool_size=[2,2])(net)\n",
    "    net = Conv2D(64, kernel_size=[5,5], strides=[1,1],activation='relu',\n",
    "                     kernel_initializer = initializers.TruncatedNormal(stddev=0.1),\n",
    "                     bias_initializer = initializers.Zeros(),\n",
    "                    padding='same')(net)\n",
    "    net = MaxPooling2D(pool_size=[2,2])(net)\n",
    "    net = Flatten()(net)\n",
    "    net = Dense(1000, activation='relu')(net) \n",
    "    net = Dense(10,activation='softmax')(net)\n",
    "\n",
    "    cross_entropy = tf.reduce_mean(categorical_crossentropy(y_, net))\n",
    "    l2_loss = tf.add_n( [tf.nn.l2_loss(w) for w in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)] )\n",
    "\n",
    "    total_loss = cross_entropy + Lambda*l2_loss\n",
    "\n",
    "    train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(total_loss)\n",
    "\n",
    "    sess = tf.Session()\n",
    "    K.set_session(sess)\n",
    "    init_op = tf.global_variables_initializer()\n",
    "    sess.run(init_op)\n",
    "    # Train\n",
    "    start = time.time()\n",
    "    for step in range(3000):\n",
    "        batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "        _, loss, l2_loss_value, total_loss_value = sess.run(\n",
    "                   [train_step, cross_entropy, l2_loss, total_loss], \n",
    "                   feed_dict={x: batch_xs, y_: batch_ys, learning_rate:lr})\n",
    "\n",
    "        if (step+1) % 100 == 0:\n",
    "            #print('step %d, entropy loss: %f, l2_loss: %f, total loss: %f' % \n",
    "             #  (step+1, loss, l2_loss_value, total_loss_value))\n",
    "            #Test trained model\n",
    "            correct_prediction = tf.equal(tf.argmax(net, 1), tf.argmax(y_, 1))\n",
    "            accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "            print(\"训练集准确度为\",sess.run(accuracy, feed_dict={x: batch_xs, y_: batch_ys}))\n",
    "        if (step+1) % 1000 == 0:\n",
    "            print(\"测试集准确度为\",sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                        y_: mnist.test.labels}))\n",
    "            end = time.time()\n",
    "            print(\"%d步所用的时间为%fs\"%(step+1,end-start))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "学习率为0.001000,L2正则系数0.00000010\n",
      "训练集准确度为 0.49\n",
      "训练集准确度为 0.57\n",
      "训练集准确度为 0.86\n",
      "训练集准确度为 0.74\n",
      "训练集准确度为 0.86\n",
      "训练集准确度为 0.88\n",
      "训练集准确度为 0.9\n",
      "训练集准确度为 0.88\n",
      "训练集准确度为 0.85\n",
      "训练集准确度为 0.87\n",
      "测试集准确度为 0.8916\n",
      "1000步所用的时间为316.719981s\n",
      "训练集准确度为 0.86\n",
      "训练集准确度为 0.86\n",
      "训练集准确度为 0.79\n",
      "训练集准确度为 0.86\n",
      "训练集准确度为 0.88\n",
      "训练集准确度为 0.92\n",
      "训练集准确度为 0.93\n",
      "训练集准确度为 0.9\n",
      "训练集准确度为 0.91\n",
      "训练集准确度为 0.92\n",
      "测试集准确度为 0.9224\n",
      "2000步所用的时间为563.349955s\n",
      "训练集准确度为 0.93\n",
      "训练集准确度为 0.97\n",
      "训练集准确度为 0.87\n",
      "训练集准确度为 0.92\n",
      "训练集准确度为 0.92\n",
      "训练集准确度为 0.93\n",
      "训练集准确度为 0.94\n",
      "训练集准确度为 0.83\n",
      "训练集准确度为 0.92\n",
      "训练集准确度为 0.95\n",
      "测试集准确度为 0.9346\n",
      "3000步所用的时间为802.340086s\n",
      "学习率为0.001000,L2正则系数0.00000100\n",
      "训练集准确度为 0.37\n",
      "训练集准确度为 0.67\n",
      "训练集准确度为 0.77\n",
      "训练集准确度为 0.82\n",
      "训练集准确度为 0.91\n",
      "训练集准确度为 0.81\n",
      "训练集准确度为 0.86\n",
      "训练集准确度为 0.92\n",
      "训练集准确度为 0.87\n",
      "训练集准确度为 0.91\n",
      "测试集准确度为 0.8919\n",
      "1000步所用的时间为245.064001s\n",
      "训练集准确度为 0.91\n",
      "训练集准确度为 0.9\n",
      "训练集准确度为 0.92\n",
      "训练集准确度为 0.92\n",
      "训练集准确度为 0.93\n",
      "训练集准确度为 0.9\n",
      "训练集准确度为 0.93\n",
      "训练集准确度为 0.92\n",
      "训练集准确度为 0.94\n",
      "训练集准确度为 0.91\n",
      "测试集准确度为 0.9197\n",
      "2000步所用的时间为488.989626s\n",
      "训练集准确度为 0.93\n",
      "训练集准确度为 0.9\n",
      "训练集准确度为 0.89\n",
      "训练集准确度为 0.91\n",
      "训练集准确度为 0.92\n",
      "训练集准确度为 0.96\n",
      "训练集准确度为 0.94\n",
      "训练集准确度为 0.93\n",
      "训练集准确度为 0.96\n",
      "训练集准确度为 0.97\n",
      "测试集准确度为 0.9345\n",
      "3000步所用的时间为733.308529s\n",
      "学习率为0.001000,L2正则系数0.00001000\n",
      "训练集准确度为 0.43\n",
      "训练集准确度为 0.57\n",
      "训练集准确度为 0.69\n",
      "训练集准确度为 0.78\n",
      "训练集准确度为 0.86\n",
      "训练集准确度为 0.89\n",
      "训练集准确度为 0.84\n",
      "训练集准确度为 0.94\n",
      "训练集准确度为 0.87\n",
      "训练集准确度为 0.92\n",
      "测试集准确度为 0.8818\n",
      "1000步所用的时间为251.385191s\n",
      "训练集准确度为 0.86\n",
      "训练集准确度为 0.89\n",
      "训练集准确度为 0.96\n",
      "训练集准确度为 0.87\n",
      "训练集准确度为 0.96\n",
      "训练集准确度为 0.94\n",
      "训练集准确度为 0.91\n",
      "训练集准确度为 0.93\n",
      "训练集准确度为 0.92\n",
      "训练集准确度为 0.9\n",
      "测试集准确度为 0.917\n",
      "2000步所用的时间为502.478888s\n",
      "训练集准确度为 0.96\n",
      "训练集准确度为 0.93\n",
      "训练集准确度为 0.89\n",
      "训练集准确度为 0.89\n",
      "训练集准确度为 0.96\n",
      "训练集准确度为 0.88\n",
      "训练集准确度为 0.96\n",
      "训练集准确度为 0.88\n",
      "训练集准确度为 0.91\n",
      "训练集准确度为 0.93\n",
      "测试集准确度为 0.9306\n",
      "3000步所用的时间为753.424480s\n",
      "学习率为0.001000,L2正则系数0.00010000\n",
      "训练集准确度为 0.45\n",
      "训练集准确度为 0.75\n",
      "训练集准确度为 0.89\n",
      "训练集准确度为 0.77\n",
      "训练集准确度为 0.82\n",
      "训练集准确度为 0.86\n",
      "训练集准确度为 0.83\n",
      "训练集准确度为 0.84\n",
      "训练集准确度为 0.89\n",
      "训练集准确度为 0.87\n",
      "测试集准确度为 0.8846\n",
      "1000步所用的时间为258.325174s\n",
      "训练集准确度为 0.89\n",
      "训练集准确度为 0.86\n",
      "训练集准确度为 0.88\n",
      "训练集准确度为 0.88\n",
      "训练集准确度为 0.94\n",
      "训练集准确度为 0.87\n",
      "训练集准确度为 0.91\n",
      "训练集准确度为 0.9\n",
      "训练集准确度为 0.9\n",
      "训练集准确度为 0.93\n",
      "测试集准确度为 0.9173\n",
      "2000步所用的时间为518.026962s\n",
      "训练集准确度为 0.95\n",
      "训练集准确度为 0.9\n",
      "训练集准确度为 0.94\n",
      "训练集准确度为 0.88\n",
      "训练集准确度为 0.95\n",
      "训练集准确度为 0.95\n",
      "训练集准确度为 0.94\n",
      "训练集准确度为 0.94\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.89\n",
      "测试集准确度为 0.9313\n",
      "3000步所用的时间为776.218682s\n",
      "学习率为0.010000,L2正则系数0.00000010\n",
      "训练集准确度为 0.9\n",
      "训练集准确度为 0.97\n",
      "训练集准确度为 0.93\n",
      "训练集准确度为 0.91\n",
      "训练集准确度为 0.9\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 0.94\n",
      "训练集准确度为 0.95\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.97\n",
      "测试集准确度为 0.9568\n",
      "1000步所用的时间为267.200779s\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.97\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.97\n",
      "测试集准确度为 0.9733\n",
      "2000步所用的时间为533.414176s\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 0.97\n",
      "训练集准确度为 0.96\n",
      "训练集准确度为 1.0\n",
      "测试集准确度为 0.9773\n",
      "3000步所用的时间为799.792690s\n",
      "学习率为0.010000,L2正则系数0.00000100\n",
      "训练集准确度为 0.94\n",
      "训练集准确度为 0.93\n",
      "训练集准确度为 0.93\n",
      "训练集准确度为 0.96\n",
      "训练集准确度为 0.97\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.96\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 0.96\n",
      "测试集准确度为 0.9578\n",
      "1000步所用的时间为272.802142s\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.97\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.97\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 0.92\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.98\n",
      "测试集准确度为 0.9755\n",
      "2000步所用的时间为545.003777s\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 0.97\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 0.97\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.99\n",
      "测试集准确度为 0.9807\n",
      "3000步所用的时间为818.256155s\n",
      "学习率为0.010000,L2正则系数0.00001000\n",
      "训练集准确度为 0.88\n",
      "训练集准确度为 0.92\n",
      "训练集准确度为 0.96\n",
      "训练集准确度为 0.96\n",
      "训练集准确度为 0.91\n",
      "训练集准确度为 0.95\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.94\n",
      "训练集准确度为 0.96\n",
      "训练集准确度为 0.97\n",
      "测试集准确度为 0.9611\n",
      "1000步所用的时间为280.630130s\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.97\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.97\n",
      "训练集准确度为 0.95\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 1.0\n",
      "测试集准确度为 0.9772\n",
      "2000步所用的时间为561.597969s\n",
      "训练集准确度为 0.96\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.99\n",
      "测试集准确度为 0.9811\n",
      "3000步所用的时间为842.302786s\n",
      "学习率为0.010000,L2正则系数0.00010000\n",
      "训练集准确度为 0.9\n",
      "训练集准确度为 0.9\n",
      "训练集准确度为 0.92\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.97\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.96\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 0.97\n",
      "训练集准确度为 0.98\n",
      "测试集准确度为 0.9602\n",
      "1000步所用的时间为287.941922s\n",
      "训练集准确度为 0.97\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.94\n",
      "训练集准确度为 0.96\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "测试集准确度为 0.9755\n",
      "2000步所用的时间为576.090843s\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 0.98\n",
      "测试集准确度为 0.9777\n",
      "3000步所用的时间为864.166713s\n",
      "学习率为0.100000,L2正则系数0.00000010\n",
      "训练集准确度为 0.94\n",
      "训练集准确度为 0.96\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "测试集准确度为 0.9878\n",
      "1000步所用的时间为295.067301s\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "测试集准确度为 0.9896\n",
      "2000步所用的时间为591.004733s\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "测试集准确度为 0.9912\n",
      "3000步所用的时间为885.930450s\n",
      "学习率为0.100000,L2正则系数0.00000100\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "测试集准确度为 0.9868\n",
      "1000步所用的时间为303.532081s\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "测试集准确度为 0.9898\n",
      "2000步所用的时间为606.529521s\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "测试集准确度为 0.9911\n",
      "3000步所用的时间为909.593876s\n",
      "学习率为0.100000,L2正则系数0.00001000\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 0.96\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 0.99\n",
      "测试集准确度为 0.9871\n",
      "1000步所用的时间为310.593747s\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "测试集准确度为 0.9875\n",
      "2000步所用的时间为620.218214s\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "测试集准确度为 0.9911\n",
      "3000步所用的时间为937.983080s\n",
      "学习率为0.100000,L2正则系数0.00010000\n",
      "训练集准确度为 0.98\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 0.99\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 0.99\n",
      "测试集准确度为 0.9884\n",
      "1000步所用的时间为319.847669s\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "测试集准确度为 0.9884\n",
      "2000步所用的时间为638.427733s\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "训练集准确度为 1.0\n",
      "测试集准确度为 0.9919\n",
      "3000步所用的时间为956.528528s\n",
      "学习率为1.000000,L2正则系数0.00000010\n",
      "训练集准确度为 0.22\n",
      "训练集准确度为 0.12\n",
      "训练集准确度为 0.13\n",
      "训练集准确度为 0.09\n",
      "训练集准确度为 0.1\n",
      "训练集准确度为 0.18\n",
      "训练集准确度为 0.08\n",
      "训练集准确度为 0.1\n",
      "训练集准确度为 0.1\n",
      "训练集准确度为 0.14\n",
      "测试集准确度为 0.1135\n",
      "1000步所用的时间为325.840722s\n",
      "训练集准确度为 0.13\n",
      "训练集准确度为 0.1\n",
      "训练集准确度为 0.05\n",
      "训练集准确度为 0.09\n",
      "训练集准确度为 0.17\n",
      "训练集准确度为 0.12\n",
      "训练集准确度为 0.17\n",
      "训练集准确度为 0.11\n",
      "训练集准确度为 0.16\n",
      "训练集准确度为 0.11\n",
      "测试集准确度为 0.1135\n",
      "2000步所用的时间为650.933096s\n",
      "训练集准确度为 0.15\n",
      "训练集准确度为 0.09\n",
      "训练集准确度为 0.13\n",
      "训练集准确度为 0.16\n",
      "训练集准确度为 0.17\n",
      "训练集准确度为 0.11\n",
      "训练集准确度为 0.07\n",
      "训练集准确度为 0.09\n",
      "训练集准确度为 0.16\n",
      "训练集准确度为 0.14\n",
      "测试集准确度为 0.1135\n",
      "3000步所用的时间为976.181569s\n",
      "学习率为1.000000,L2正则系数0.00000100\n",
      "训练集准确度为 0.11\n",
      "训练集准确度为 0.08\n",
      "训练集准确度为 0.15\n",
      "训练集准确度为 0.1\n",
      "训练集准确度为 0.09\n",
      "训练集准确度为 0.12\n",
      "训练集准确度为 0.16\n",
      "训练集准确度为 0.1\n",
      "训练集准确度为 0.17\n",
      "训练集准确度为 0.08\n",
      "测试集准确度为 0.1135\n",
      "1000步所用的时间为333.600463s\n",
      "训练集准确度为 0.11\n",
      "训练集准确度为 0.08\n",
      "训练集准确度为 0.11\n",
      "训练集准确度为 0.08\n",
      "训练集准确度为 0.12\n",
      "训练集准确度为 0.15\n",
      "训练集准确度为 0.09\n",
      "训练集准确度为 0.13\n",
      "训练集准确度为 0.08\n",
      "训练集准确度为 0.17\n",
      "测试集准确度为 0.1032\n",
      "2000步所用的时间为666.384819s\n",
      "训练集准确度为 0.15\n",
      "训练集准确度为 0.13\n",
      "训练集准确度为 0.12\n",
      "训练集准确度为 0.1\n",
      "训练集准确度为 0.14\n",
      "训练集准确度为 0.08\n",
      "训练集准确度为 0.11\n",
      "训练集准确度为 0.11\n",
      "训练集准确度为 0.14\n",
      "训练集准确度为 0.1\n",
      "测试集准确度为 0.1028\n",
      "3000步所用的时间为999.785549s\n",
      "学习率为1.000000,L2正则系数0.00001000\n",
      "训练集准确度为 0.11\n",
      "训练集准确度为 0.18\n",
      "训练集准确度为 0.14\n",
      "训练集准确度为 0.16\n",
      "训练集准确度为 0.15\n",
      "训练集准确度为 0.11\n",
      "训练集准确度为 0.15\n",
      "训练集准确度为 0.11\n",
      "训练集准确度为 0.13\n",
      "训练集准确度为 0.22\n",
      "测试集准确度为 0.1028\n",
      "1000步所用的时间为351.632802s\n",
      "训练集准确度为 0.11\n",
      "训练集准确度为 0.08\n",
      "训练集准确度为 0.11\n",
      "训练集准确度为 0.13\n",
      "训练集准确度为 0.13\n",
      "训练集准确度为 0.1\n",
      "训练集准确度为 0.12\n",
      "训练集准确度为 0.13\n",
      "训练集准确度为 0.14\n",
      "训练集准确度为 0.12\n",
      "测试集准确度为 0.1135\n",
      "2000步所用的时间为692.804016s\n",
      "训练集准确度为 0.09\n",
      "训练集准确度为 0.12\n",
      "训练集准确度为 0.03\n",
      "训练集准确度为 0.1\n",
      "训练集准确度为 0.18\n",
      "训练集准确度为 0.15\n",
      "训练集准确度为 0.13\n",
      "训练集准确度为 0.15\n",
      "训练集准确度为 0.11\n",
      "训练集准确度为 0.1\n",
      "测试集准确度为 0.1135\n",
      "3000步所用的时间为1032.822111s\n",
      "学习率为1.000000,L2正则系数0.00010000\n",
      "训练集准确度为 0.21\n",
      "训练集准确度为 0.07\n",
      "训练集准确度为 0.06\n",
      "训练集准确度为 0.15\n",
      "训练集准确度为 0.13\n",
      "训练集准确度为 0.08\n",
      "训练集准确度为 0.08\n",
      "训练集准确度为 0.15\n",
      "训练集准确度为 0.13\n",
      "训练集准确度为 0.17\n",
      "测试集准确度为 0.098\n",
      "1000步所用的时间为350.558964s\n",
      "训练集准确度为 0.11\n",
      "训练集准确度为 0.11\n",
      "训练集准确度为 0.11\n",
      "训练集准确度为 0.14\n",
      "训练集准确度为 0.11\n",
      "训练集准确度为 0.11\n",
      "训练集准确度为 0.16\n",
      "训练集准确度为 0.14\n",
      "训练集准确度为 0.13\n",
      "训练集准确度为 0.15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "测试集准确度为 0.098\n",
      "2000步所用的时间为700.638132s\n",
      "训练集准确度为 0.05\n",
      "训练集准确度为 0.12\n",
      "训练集准确度为 0.19\n",
      "训练集准确度为 0.11\n",
      "训练集准确度为 0.1\n",
      "训练集准确度为 0.04\n",
      "训练集准确度为 0.12\n",
      "训练集准确度为 0.13\n",
      "训练集准确度为 0.09\n",
      "训练集准确度为 0.09\n",
      "测试集准确度为 0.1135\n",
      "3000步所用的时间为1050.683756s\n"
     ]
    }
   ],
   "source": [
    "lrs = [0.001,0.01,0.1,1]\n",
    "Lambdas = [1e-7,1e-6,1e-5,1e-4]\n",
    "for lr in lrs:\n",
    "    for Lambda in Lambdas:\n",
    "        print(\"学习率为%f,L2正则系数%.8f\"%(lr,Lambda))\n",
    "        CONV(lr = lr, Lambda = Lambda)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "由上述分析可以看出，当学习率为0.1，正则参数为0.0001时，测试集准确度最高，达到99.2%"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "综合上述几点分析可知，在手写字体识别中，使用两层卷积，两层池化，并且卷积核权重服从截断正态分布，\n",
    "能够明显提高识别准确度，同时通过调整学习率及L2正则参数，最终测试集准确度达到99.2%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
